# 词频数据处理脚本
# Word Frequency Data Processing Script

# Clean workspace
rm(list = ls())  # Remove all objects
graphics.off()    # Close all graphics devices
cat("\014")      # Clear console

# 加载必要的包
if (!require(tidyverse)) install.packages("tidyverse")
if (!require(readxl)) install.packages("readxl")
if (!require(writexl)) install.packages("writexl")

library(tidyverse)
library(readxl)
library(writexl)

# 设置输入文件路径
input_file <- "source_data/Ch_Ja_Lex_Behavior_2Group_29Sub_for R.xlsx"
word_freq_file <- "source_data/日语词频计算.xlsx"

# 创建结果文件夹
result_path <- "results/word_frequency"
if (!dir.exists(result_path)) {
  dir.create(result_path, recursive = TRUE)
  print(paste("Created result folder:", result_path))
}

# 加载行为数据
print("Loading behavior data...")
behavior_data <- read_excel(input_file, sheet = 3)
print("Behavior data loaded")

# 处理变量名中的空格
names(behavior_data) <- gsub(" ", "_", names(behavior_data))
print("Variable names processed (spaces replaced with underscores)")

# 检查行为数据的列名
print("Behavior data columns:")
print(names(behavior_data))

# 检查是否有Prime_word和Target_word列
if (!("Prime_word" %in% names(behavior_data)) || !("Target_word" %in% names(behavior_data))) {
  stop("Error: Behavior data does not contain required columns 'Prime_word' and 'Target_word'")
}

# 获取所有出现在行为数据中的词
all_behavior_words <- unique(c(behavior_data$Prime_word, behavior_data$Target_word))
print(paste("Number of unique words found:", length(all_behavior_words)))

# 创建词频数据框
word_freq_processed <- data.frame(
  Word = all_behavior_words,
  Chinese_Freq = rep(NA, length(all_behavior_words)),
  Japanese_Freq = rep(NA, length(all_behavior_words)),
  Source = rep(NA, length(all_behavior_words)),
  stringsAsFactors = FALSE
)

# 加载词频数据
print("Loading word frequency data...")
word_freq_data <- read_excel(word_freq_file, sheet = 2)
print("Word frequency data loaded")

# 重命名列以便于使用
names(word_freq_data) <- c("Word", "Chinese_Freq", "Japanese_Freq")

# 计算词频统计量，用于估计
word_stats <- list(
  Chinese = list(
    median = median(word_freq_data$Chinese_Freq, na.rm = TRUE),
    mean = mean(word_freq_data$Chinese_Freq, na.rm = TRUE),
    sd = sd(word_freq_data$Chinese_Freq, na.rm = TRUE),
    q25 = quantile(word_freq_data$Chinese_Freq, 0.25, na.rm = TRUE),
    q75 = quantile(word_freq_data$Chinese_Freq, 0.75, na.rm = TRUE)
  ),
  Japanese = list(
    median = median(word_freq_data$Japanese_Freq, na.rm = TRUE),
    mean = mean(word_freq_data$Japanese_Freq, na.rm = TRUE),
    sd = sd(word_freq_data$Japanese_Freq, na.rm = TRUE),
    q25 = quantile(word_freq_data$Japanese_Freq, 0.25, na.rm = TRUE),
    q75 = quantile(word_freq_data$Japanese_Freq, 0.75, na.rm = TRUE)
  )
)

# 设置随机种子（只设置一次）
set.seed(123)

# 匹配词频数据
for(i in 1:nrow(word_freq_processed)) {
  word <- word_freq_processed$Word[i]
  freq_idx <- match(word, word_freq_data$Word)
  
  if(!is.na(freq_idx)) {
    # 如果在词频表中找到匹配
    word_freq_processed$Chinese_Freq[i] <- word_freq_data$Chinese_Freq[freq_idx]
    word_freq_processed$Japanese_Freq[i] <- word_freq_data$Japanese_Freq[freq_idx]
    word_freq_processed$Source[i] <- "词频表"
  } else {
    # 如果在词频表中未找到匹配，使用随机数估计词频
    # 使用词频数据的统计特征生成随机数
    # 确保生成的随机数不与已有词频重复
    repeat {
      estimated_chinese_freq <- rnorm(1, mean = word_stats$Chinese$mean, sd = word_stats$Chinese$sd)
      estimated_japanese_freq <- rnorm(1, mean = word_stats$Japanese$mean, sd = word_stats$Japanese$sd)
      
      # 确保估计值在合理范围内
      estimated_chinese_freq <- max(min(estimated_chinese_freq, word_stats$Chinese$q75), word_stats$Chinese$q25)
      estimated_japanese_freq <- max(min(estimated_japanese_freq, word_stats$Japanese$q75), word_stats$Japanese$q25)
      
      # 检查是否与已有词频重复
      if (!estimated_chinese_freq %in% word_freq_processed$Chinese_Freq[!is.na(word_freq_processed$Chinese_Freq)] &&
          !estimated_japanese_freq %in% word_freq_processed$Japanese_Freq[!is.na(word_freq_processed$Japanese_Freq)]) {
        break
      }
    }
    
    word_freq_processed$Chinese_Freq[i] <- estimated_chinese_freq
    word_freq_processed$Japanese_Freq[i] <- estimated_japanese_freq
    word_freq_processed$Source[i] <- "随机估计"
  }
}

# 对词频进行对数转换
word_freq_processed$Log_Chinese_Freq <- log(word_freq_processed$Chinese_Freq)
word_freq_processed$Log_Japanese_Freq <- log(word_freq_processed$Japanese_Freq)

# 计算词频比例（相对于语料库总条数）
chinese_corpus_total <- 1200000000  # 12亿
japanese_corpus_total <- 1100000000  # 11亿

word_freq_processed$Chinese_Freq_Prop <- word_freq_processed$Chinese_Freq / chinese_corpus_total
word_freq_processed$Japanese_Freq_Prop <- word_freq_processed$Japanese_Freq / japanese_corpus_total

# 对词频比例进行对数转换
word_freq_processed$Log_Chinese_Freq_Prop <- log(word_freq_processed$Chinese_Freq_Prop)
word_freq_processed$Log_Japanese_Freq_Prop <- log(word_freq_processed$Japanese_Freq_Prop)

# 进行z-score标准化
word_freq_processed$Z_Chinese_Freq <- scale(word_freq_processed$Log_Chinese_Freq)
word_freq_processed$Z_Japanese_Freq <- scale(word_freq_processed$Log_Japanese_Freq)
word_freq_processed$Z_Chinese_Freq_Prop <- scale(word_freq_processed$Log_Chinese_Freq_Prop)
word_freq_processed$Z_Japanese_Freq_Prop <- scale(word_freq_processed$Log_Japanese_Freq_Prop)

# 打印词频处理结果
print("\n词频数据处理结果:")
print("中文词频数据基本统计:")
print(summary(word_freq_processed$Chinese_Freq))
print("\n日文词频数据基本统计:")
print(summary(word_freq_processed$Japanese_Freq))
print("\n中文词频比例统计:")
print(summary(word_freq_processed$Chinese_Freq_Prop))
print("\n日文词频比例统计:")
print(summary(word_freq_processed$Japanese_Freq_Prop))
print("\n中文对数转换后的词频统计:")
print(summary(word_freq_processed$Log_Chinese_Freq))
print("\n日文对数转换后的词频统计:")
print(summary(word_freq_processed$Log_Japanese_Freq))
print("\n中文对数转换后的词频比例统计:")
print(summary(word_freq_processed$Log_Chinese_Freq_Prop))
print("\n日文对数转换后的词频比例统计:")
print(summary(word_freq_processed$Log_Japanese_Freq_Prop))
print("\n中文Z-score 统计:")
print(summary(word_freq_processed$Z_Chinese_Freq))
print("\n日文Z-score 统计:")
print(summary(word_freq_processed$Z_Japanese_Freq))
print("\n中文词频比例Z-score 统计:")
print(summary(word_freq_processed$Z_Chinese_Freq_Prop))
print("\n日文词频比例Z-score 统计:")
print(summary(word_freq_processed$Z_Japanese_Freq_Prop))
print("\n来源统计:")
print(table(word_freq_processed$Source))

# 打印每种来源的词频分布
print("\n各来源中文词频分布:")
tapply(word_freq_processed$Chinese_Freq, word_freq_processed$Source, summary)
print("\n各来源日文词频分布:")
tapply(word_freq_processed$Japanese_Freq, word_freq_processed$Source, summary)

# 保存词频数据到文件
freq_result_file <- file.path(result_path, "word_frequency_processed.xlsx")
write_xlsx(word_freq_processed, freq_result_file)
print(paste("词频处理结果已保存到:", freq_result_file))

print("\n============= Word Frequency Processing Completed =============") 